package crawler;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Random;

import net.GooglePageParserRegexp;
import net.GoogleResultItem;
import net.PageFetcher;

public class CrawlGoogleResults {

	public static void main(String args[]) throws IOException {
		
		ArrayList<String> API_List = getAPI_List(0);
		Random random = new Random();	
		
		for(String API_FQN:API_List) {
			//read from file
			File dir = new File("D://Code Snippet Reuse/test_data/"+API_FQN+"/");
			if(!dir.exists()){
				dir.mkdirs();
			}
			
			ArrayList<GoogleResultItem> googleResults = new ArrayList<GoogleResultItem>();
			int GooglePages = 10;
			// Get top 50 results
			
			String query = "%22"+API_FQN+"%22+example";
			
			boolean shouldContinue = true;
			
			for (int i = 0; i < GooglePages; i++) {
				if(!shouldContinue){
					break;
				}
				System.err.println("Getting google result in pp.:" + i + "...");
				
				String queryURL = "http://www.google.com/search?q="
						+ query + "&hl=en&start="
						+ (i * 10) + "&sa=N";
				System.out.println(queryURL);
				String content = PageFetcher.fetchURL(queryURL);
				//record google result page
				writeFile("D://Code Snippet Reuse/test_data/"+API_FQN+"/google_page_"
							+ i + ".html", content);
				
				System.err.println("Got it! Size:"+content.length());
				
				ArrayList<GoogleResultItem> items = new GooglePageParserRegexp()
						.parse(content, "pp_" + i);
				
				shouldContinue = false;
				
				for (GoogleResultItem googleResultItem : items) {
					if (!googleResults.contains(googleResultItem)) {
						
						shouldContinue = true;
						
						googleResults.add(googleResultItem);
						System.out.println("BackLink:\t"
								+ googleResultItem.getOriginalURL());
						System.out.println("Anchor:\t"
								+ googleResultItem.getAnchorText());
						System.out.println("Summary:\t"
								+ googleResultItem.getSummary());
					}
				}
				System.err.println("Waiting for 5 seconds...");
				try {
					Thread.sleep(5000);
				} catch (InterruptedException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
			
			int num = 0;			
			PrintWriter pw = new PrintWriter(new FileWriter(new File("D://Code Snippet Reuse/test_data/"+API_FQN+"/backlinks.txt")));
			for (GoogleResultItem googleResultItem : googleResults) {
				pw.println(num++ +"\t"+googleResultItem.getOriginalURL());
			}
			pw.flush();
			pw.close();
			//Get google results for next API, sleep 10~25 seconds first
			try {
				Thread.sleep(10000+random.nextInt(15)*1000);
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
	}
	//read API list from file
	public static ArrayList<String> getAPI_List(int startIndex) throws IOException{
		BufferedReader br = new BufferedReader(new FileReader(new File(
				"D://Code Snippet Reuse/jdkClassList.txt")));
		String line = null;
		ArrayList<String> API_List = new ArrayList<String>();
		int index = 0;
		while ((line = br.readLine()) != null) {
			if((index++)>=startIndex){
				API_List.add(line.trim());
			}
		}
		return API_List;
	}
	
	private static void writeFile(String fileName,String content){
		try {
			PrintWriter pw = new PrintWriter(new FileWriter(new File(fileName)));
			pw.println(content);
			pw.flush();
			pw.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

}
